#include "parser.h"

void unify_url(char *src, char *url, char *c_url)
{
	char		*r, *p, *q, has_host;

	//处理http://
	has_host = 0;
	strcpy(url, "http://");
	if (strstr(src, "http://")) {
		has_host = 1;
		src += 7;
	}
	//处理host
	//如果不以http://开头，而且'.'后面没有'/'或者'/'前面没有'.', 则src没有host
	if (!has_host) {
		strcat(url, url_get_host(c_url));
	} else {
		p = url + strlen(url);
		for (; *src && *src != '/'; *p++ = *src++);
		*p = '\0';
	}
	//c_url = "http://host.com/data/structure/index.html"
	//c_url = "http://host.com/data/index.html"
	//c_url = "http://host.com/index.html"
	//此时url = "http://hostname.com"  src = "/foo/bar/index.html"或"foo/bar/index.html"
	//或"./foo/index.html"或../index.html或/foo或../foo
	//处理目录
	if (has_host) {
		strcat(url, src);
	}
	else {
		for (p = c_url+7; *p != '/'; p++);
		for (q = c_url+strlen(c_url); *(q-1) != '/'; q--);
		if (src[0] == '.' && src[1] == '.') {
			if ((q-p) > 1) {
				for (q-=2; *(q-1) != '/'; q--);
			}
			strncat(url, p, q-p);
			src += 2;
			if (*src == '/') src++;
		}
		else if (src[0] == '.') {
			strncat(url, p, q-p);
			src++;
			if (*src == '/') src++;
		}
		else if (src[0] != '/') {
			strncat(url, p, q-p);
		}
		strcat(url, src);
	}
	//处理#号
	p = strchr(url, '#');
	if (p) *p = '\0';

	//处理文件
	if (strstr(url, ".htm") == NULL) {
		if (url[strlen(url)-1] == '/') {
			strcat(url, "index.html");
		} else {
			strcat(url, "/index.html");
		}
	}
	//把htm变成html
	r = url + strlen(url) - 1;
	if (*r != 'l') {
		*(++r) = 'l';
		*(++r) = '\0';
	}
}

void parse_html(global_t *global, conn_t *conn)
{	
	char				*p, *q;
	char				href_str[MAX_URL];
	char				url[MAX_URL];

	p = conn->buffer;
	while ((p = strcasestr(p, "a href=")) != NULL) {
		p += strlen("<a href=");
		if (*p == '"') p++;
		q = p;
		for (; *p != '"' && *p != '>'; p++);
		if (q-p > MAX_URL) q = p + MAX_URL - 3;
		strncpy(href_str, q, p-q);
		href_str[p-q-1] = '\0';
		//printf("%s   %s\n", href_str, conn->url);
		if (strstr(href_str, "mailto") || strstr(href_str, "ftp://")) continue;

		unify_url(href_str, url, conn->url);
		//printf("%s\n", url);

		//如果之前没有索引过，入队，如果主机没有dns，查询dns
		if (ht_insert(global->url_table, url, NULL)) {
			if (ht_find(global->host_table, url_get_host(url)) == NULL) {
				if(strstr(url, "twitter")) continue;
				debug("查询%s\n", url);
				dns_request(global->dns, 
						url_get_host(url), DNS_A_RECORD);
			}
			q_push(global->url_queue, strdup(url));
			sem_post(&global->usem);
			//printf("把%s加入url_queue, size = %d\n", url, q_size(global->url_queue));
		}
	}	
}
